In [12]:

from zfish.multi_table.tables_io import scan_resources
import polars as pl

IMG_ID = ["m", "roi", "c"]

r = scan_resources().collect() # load resources tables

rois = r.rois["roi"][:2]
channels = ["DAPI.0", "DAPI.1"]
id_columns = IMG_ID

df_imgs = (
    r.images  # extract images table  
    .filter(pl.col("m") < 2)  # select top 2 reslutions
    .filter(pl.col("roi").is_in(rois))  # select 2 rois
    .filter(pl.col("c").is_in(channels)) # select 2 channels
)

for img in df_imgs.select(id_columns).rows(named=True):
    print("Image ID:", img)

Image ID: {'m': 0, 'roi': 'B02_px+0198_py-2152', 'c': 'DAPI.0'}
Image ID: {'m': 0, 'roi': 'B02_px+0198_py-2152', 'c': 'DAPI.1'}
Image ID: {'m': 1, 'roi': 'B02_px+0198_py-2152', 'c': 'DAPI.0'}
Image ID: {'m': 1, 'roi': 'B02_px+0198_py-2152', 'c': 'DAPI.1'}
Image ID: {'m': 0, 'roi': 'B02_px+0385_py-0060', 'c': 'DAPI.0'}
Image ID: {'m': 0, 'roi': 'B02_px+0385_py-0060', 'c': 'DAPI.1'}
Image ID: {'m': 1, 'roi': 'B02_px+0385_py-0060', 'c': 'DAPI.0'}
Image ID: {'m': 1, 'roi': 'B02_px+0385_py-0060', 'c': 'DAPI.1'}

If we want to do the same thing but in batches per roi to send each job to a separate worker on an HPC cluster:

In [13]:

batch_columns = ["roi"]
id_columns = [e for e in id_columns if e not in batch_columns]

i = 0
for name_batch, df_batch in df_imgs.group_by(batch_columns):
    print(f"Batch {i}:")
    print("  Batch ID:", dict(zip(batch_columns, name_batch)))
    for img in df_batch.select(id_columns).rows(named=True):
        print("    Image ID:", img)
    i += 1

Batch 0:
  Batch ID: {'roi': 'B02_px+0385_py-0060'}
    Image ID: {'m': 0, 'c': 'DAPI.0'}
    Image ID: {'m': 0, 'c': 'DAPI.1'}
    Image ID: {'m': 1, 'c': 'DAPI.0'}
    Image ID: {'m': 1, 'c': 'DAPI.1'}
Batch 1:
  Batch ID: {'roi': 'B02_px+0198_py-2152'}
    Image ID: {'m': 0, 'c': 'DAPI.0'}
    Image ID: {'m': 0, 'c': 'DAPI.1'}
    Image ID: {'m': 1, 'c': 'DAPI.0'}
    Image ID: {'m': 1, 'c': 'DAPI.1'}

If there’s even more free resources on the cluster and we’re in a hurry:

In [14]:

batch_columns = ["roi", "m"]
id_columns = [e for e in id_columns if e not in batch_columns]

i = 0
for name_batch, df_batch in df_imgs.group_by(batch_columns):
    print(f"Batch {i}:")
    print(f"  Batch ID:", dict(zip(batch_columns, name_batch)))
    for img in df_batch.select(id_columns).rows(named=True):
        print("    Image ID:", img)
    i += 1

Batch 0:
  Batch ID: {'roi': 'B02_px+0198_py-2152', 'm': 1}
    Image ID: {'c': 'DAPI.0'}
    Image ID: {'c': 'DAPI.1'}
Batch 1:
  Batch ID: {'roi': 'B02_px+0385_py-0060', 'm': 0}
    Image ID: {'c': 'DAPI.0'}
    Image ID: {'c': 'DAPI.1'}
Batch 2:
  Batch ID: {'roi': 'B02_px+0385_py-0060', 'm': 1}
    Image ID: {'c': 'DAPI.0'}
    Image ID: {'c': 'DAPI.1'}
Batch 3:
  Batch ID: {'roi': 'B02_px+0198_py-2152', 'm': 0}
    Image ID: {'c': 'DAPI.0'}
    Image ID: {'c': 'DAPI.1'}

As you can see it is trivial to process the data by walking along the coordinates, and batching can be done flexibly by grouping by the respective columns.